package fetcher

import (
    "bufio"
    "golang.org/x/net/html/charset"
    "golang.org/x/text/encoding"
    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
    "io"
    "io/ioutil"
    "net/http"
    "net/http/cookiejar"
)

const UserAgent = `Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0`

var jar, _ = cookiejar.New(nil)

func FetchData(url string) ([]byte, error) {
    client := http.Client{
        CheckRedirect: func(req *http.Request, via []*http.Request) error {
            return nil
        },
        Jar: jar,
    }
    
    req, _ := http.NewRequest(http.MethodGet, url, nil)
    
    req.Header.Add("User-Agent", UserAgent)
    
    resp, err := client.Do(req)
    
    if err != nil {
        return nil, err
    }
    
    defer resp.Body.Close()
    
    return readUtf8(resp.Body)
}

func readUtf8(reader io.Reader) ([]byte, error) {
    br := bufio.NewReader(reader)
    
    //取出1024字节数据分析选用哪种编码器
    var e encoding.Encoding
    data, err := br.Peek(1024)
    if err != nil {
        e = unicode.UTF8
    } else {
        e, _, _ = charset.DetermineEncoding(data, "")
    }
    
    r := transform.NewReader(br, e.NewDecoder())
    
    return ioutil.ReadAll(r)
}
